# -*- coding: utf-8 -*-
"""
Created on Fri Nov 17 14:56:10 2017

@author: JanComputer
"""
import requests
import logging
from urllib.parse import urlparse
class Crawl():
    def __init__(self):
        self.headers={'user-agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}
        self._netloc=None
    def crawlHtml(self,url):
        try:
            parse_result=urlparse(url)
            self._netloc=parse_result.scheme+'://'+parse_result.netloc
            r=requests.get(url,headers=self.headers)
            r.encoding='utf-8'
            text=r.text
            return {'netloc':self._netloc,'text':text}
        except Exception as e:
            logging.exception('请求有误...')
        return None
        